{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/news/fake-news/fake-news-negatives-summarization.json\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/news/fake-news/fake-news-positives-summarization.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/compressed-fake-news.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/250-news-with-valid-hoax-label-summaries.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/600-news-with-valid-hoax-label-summaries.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/facts-summaries.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/hoax-summaries.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/malaysia-scraping-syazanihussin-summaries.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/indonesian/hoax.csv\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/indonesian/facts.csv\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/indonesian/250%20news%20with%20valid%20hoax%20label.csv\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/indonesian/600%20news%20with%20valid%20hoax%20label.csv\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/malaysia-scraping-syazanihussin.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    string = unidecode(string)\n",
    "    string = string.replace('SPPPPLIIIT>', ' ').replace('SPPPPLIIIT', ' ')\n",
    "    string = re.sub(r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', '', string)\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('250 news with valid hoax label.csv', sep = ';',encoding = \"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Y = [], []\n",
    "\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,0])\n",
    "    Y.append(df.iloc[i,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('600 news with valid hoax label.csv', sep = ';',encoding = \"ISO-8859-1\")\n",
    "\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,0])\n",
    "    Y.append(df.iloc[i,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('250-news-with-valid-hoax-label-summaries.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('600-news-with-valid-hoax-label-summaries.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('facts.csv', sep = ',',encoding = \"ISO-8859-1\")\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,1])\n",
    "    Y.append(df.iloc[i,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('hoax.csv', sep = ',',encoding = \"ISO-8859-1\")\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,1])\n",
    "    Y.append(df.iloc[i,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('hoax-summaries.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('facts-summaries.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('malaysia-scraping-syazanihussin.csv')\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,0])\n",
    "    Y.append(df.iloc[i,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('malaysia-scraping-syazanihussin-summaries.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6657, 6657)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X), len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !unzip compressed-fake-news.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "\n",
    "X_, Y_ = [], []\n",
    "\n",
    "for file in glob('positive/*.json'):\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    X_.extend(data)\n",
    "    Y_.extend(['fake'] * len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in glob('negative/*.json'):\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    X_.extend(data)\n",
    "    Y_.extend(['real'] * len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('fake-news-negatives-summarization.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X_.extend(i)\n",
    "    Y_.extend(['real'] * len(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('fake-news-positives-summarization.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X_.extend(i)\n",
    "    Y_.extend(['fake'] * len(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(59935, 59935)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_), len(Y_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Jakarta, Di jejaring sosial, banyak beredar informasi yang menyebut lele sebagai ikan paling jorok. Dalam sesuap daging ikan lele, terkandung 3000 sel kanker. Benarkah? Julukan sebagai ikan paling jorok merujuk pada sifat lele yang doyan mengonsumsi segala jenis limbah di perairan. Bahkan sebuah artikel yang cukup viral di internet menyebutkan kotoran manusia juga dijadikan pakan pada sebuah budidaya lele di Kota Haikou, China. Sementara itu di habitat aslinya, lele atau catfish juga dikenal sebagai spesies ikan yang sangat tangguh. Ikan ini dilengkapi alat pernapasan tambahan berupa labirin, sehingga mampu bertahan hidup dalam kondisi perairan berlumpur atau bahkan tercemar. Agaknya, fakta inilah yang memunculkan dugaan soal akumulasi racun karsinogen (penyebab kanker) di tubuh ikan lele. Untungnya, ikan lele yang beredar di pasaran bukan berasal dari alam liar. Lele banyak dibudidayakan di kolam-kolam, yang mestinya bisa dikendalikan agar bebas dari pencemaran. Pakan yang diberikan juga bisa dipilih, tidak harus mengandalkan limbah. Yang pasti, popularitas ikan bersungut ini tidak pernah pudar, bahkan terus meningkat. Data Kementerian Kelautan dan Perikanan (KKP) menyebut produksi lele pada 2013 mencapai 543,461 ton, meningkat dari 441,217 ton pada 2012 dan 337,577 ton pada 2011. Konsumsi ikan lele menurut Badan Pusat Statistik (BPS) tercatat 29,98 kg / kapita / tahun, naik dari 22,58 kg / kapita / tahun pada 2004. Di Jakarta, tak kurang dari 6000 lapak pecel lele telah terdaftar di Asosiasi Pedagang Kaki Lima Indonesia (APKLI). Soal kandungan nutrisi, tak bisa dipungkiri bahwa lele adalah sumber protein berharga yang murah meriah. Fakta bahwa ikan lele juga rendah kolesterol sepertinya bakal menenggelamkan tudingan bahwa lele bisa memicu kanker. \"Saat ini belum ada penelitian yang menyatakan jika memakan lele dapat memicu kanker,\" tegas dr Dradjat R Suardi, SpB(K)Onk, ahli kanker dari Perhimpunan Onkologi Indonesia saat dihubungi detikHealth, Jumat (23/10/2015).'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaning(X[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 59935/59935 [00:17<00:00, 3380.69it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(50284, 50284)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "selected_X, selected_Y = [], []\n",
    "mapping_label = {'fake': 0, 'hoax': 0, 'real': 1, 'valid': 1, 'fact': 1}\n",
    "\n",
    "for i in tqdm(range(len(X_))):\n",
    "    try:\n",
    "        t = cleaning(X_[i])\n",
    "        if len(t) > 100:\n",
    "            selected_X.append(t)\n",
    "            selected_Y.append(mapping_label[Y_[i].lower()])\n",
    "    except:\n",
    "        pass\n",
    "        \n",
    "len(selected_X), len(selected_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45255, 5029)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(selected_X, selected_Y, test_size = 0.1)\n",
    "len(train_X), len(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([17321, 27934]))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.unique(train_Y, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([1990, 3039]))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(test_Y, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6657/6657 [00:01<00:00, 6192.64it/s]\n"
     ]
    }
   ],
   "source": [
    "selected_X, selected_Y = [], []\n",
    "mapping_label = {'fake': 0, 'hoax': 0, 'real': 1, 'valid': 1, 'fact': 1}\n",
    "\n",
    "for i in tqdm(range(len(X))):\n",
    "    try:\n",
    "        t = cleaning(X[i])\n",
    "        if len(t) > 100:\n",
    "            selected_X.append(t)\n",
    "            selected_Y.append(mapping_label[Y[i].lower()])\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = train_X + selected_X\n",
    "train_Y = train_Y + selected_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "train_X, train_Y = shuffle(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([19477, 32284]))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(train_Y, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset.json', 'w') as fopen:\n",
    "    json.dump({'train_X': train_X, 'train_Y': train_Y,\n",
    "              'test_X': test_X, 'test_Y': test_Y}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
